package main;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

//The HTML page as a Strin
public class Main {
	private static  String myUrl;
	private static String Trimmed(String the_string){
		 String trimmedString = "";
		int temp =  the_string.indexOf('#');
		 if (temp >= 0){
			 trimmedString =  the_string.substring(0,temp);
			 the_string =  trimmedString;
		 }
		 
		 
		 return the_string;
	 }
	private static	boolean isValid1(String the_OuterLink, String the_link){
		StringBuffer theLink = new StringBuffer();
	 	
	 		theLink.append(myUrl);
	 		theLink.append(the_OuterLink);
	 		
			String  myString = theLink.toString();
	
			boolean flag = false;
			if (myString.matches(the_link)){
				myString = Trimmed(myString);
			System.out.print(myString);
				if (myString.endsWith("html")|| myString.endsWith("/")){
					
					flag = true;
				}
			
			} 
			
			return flag;
		}
private	static boolean isValid(String the_link){
	boolean flag = true;
	if(the_link.matches("http://questioneverything.typepad.com/")){
	flag = false;
	
	}else if(the_link.startsWith("http://www.youtube.com")){
	flag = false;
	
	} else if(!the_link.startsWith("http:")){
		flag = false;
		
	} else if (the_link.endsWith(".m4v")){//||the_link.endsWith("/")){
		flag = false;
		}/*
	} else if(!the_link.endsWith("htm")||!the_link.endsWith("txt")){
		flag = false;
	*/

	return flag;
}




	    public static void main(String[] args) throws IOException {
	   // System.out.println("";)
	    	
	    	String url = "http://faculty.washington.edu/gmobus/";
	    	//String url ="http://kcts9.org/education/science-cafe/energy-economy";
	    	//String url = "http://video.kcts9.org/rss/";
	       Spider blackWidow = new Spider(url);
	       blackWidow.Run();
	    }
}
	   

	/*
	       String url = "http://faculty.washington.edu/gmobus/";
	        Document doc = Jsoup.connect(url).get();
	        Elements links = doc.select("a[href]");
	       
	        for (Element link : links) {
	        	String the_link = link.attr("abs:href");
	        
	        //	System.out.println(link.attr("abs:href"));      	
	        	if (!the_link.startsWith("http://questioneverything")){
	        	//if (the_link.endsWith(".html")||the_link.endsWith("/")||the_link.endsWith(".txt")||the_link.endsWith(".htm")){
	    			System.out.println(the_link);
	    			}
	        }
	    }
	        }
	    		/*
	        	
	        	}
	        	
	        //	if (isValid(link.attr("href"),link.attr("abs:href"))) {
	        	System.out.println(link.attr("abs:href"));
	        	//	if (isValid(link.attr("abs:href"))) {
	        	//		System.out.println(link.attr("abs:href"));      	
	        //	System.out.println(link.attr("abs:href"));
	        //	}
	        	
	     
	       }
	       /* 
	        Elements descriptions = doc.select("p");
       
            for (Element element : descriptions) {
                System.out.println(element.ownText());
               
            }
            Elements title = doc.select("title");
            
            for (Element element : title) {
                System.out.println(element.ownText());
               
            }

	    }

	
	    }
	*/
